import multiprocessing
from multiprocessing import Pool
from datetime import datetime
import platform
import subprocess
import threading
import resource
import inspect
import select
import getopt
import json
import time
import sys
import os
import re

g_kernel_thread_names = [
	'kthreadd',
	'migration',
	'ksoftirqd',
	'kworker',
	'flush',
	'kswapd',
]
g_main_argv = {}
g_softirq_counter = {}
g_back_collector = None

class CmdOutputCollector(threading.Thread):
    def __init__(self):
        super().__init__()
        self.locker = threading.Lock()
        self.tag_results  = {}
        self.tag_handlers = {}

    def add_handle(self, tag, cmd, handler):
        if tag in self.tag_handlers:
            return False, 'exist tag:{}'.format(tag)
        fd = os.popen(cmd, 'r')
        if fd is None:
            return False, 'popen fail'
        self.tag_handlers[tag] = { 
            'fd':  fd, 
			'cmd': cmd,
            'handler': handler
        }   
        return True, ''

    def process_handler(self, tag):
        data_result = ''
        value = None 
        self.locker.acquire()  
        value = self.tag_handlers[tag]
        self.locker.release()  
        fd = value['fd']
        handler = value['handler']
        while True:
            readable, _, _ = select.select([fd], [], [], 0)
            if fd in readable:
                data = os.read(fd.fileno(), 1024).decode()
                data_result += data
            else:
                break
        if len(data_result) > 0:
            is_first_result = True
            self.locker.acquire()
            if tag in self.tag_results:
                is_first_result = False
            self.locker.release()
            data_result = handler.process(data_result, opt={'first_result': is_first_result})
            self.locker.acquire()
           # if tag not in self.tag_results or self.tag_results[tag] is None:
           #     self.tag_results[tag] = data_result
            self.tag_results[tag] = data_result
            self.locker.release()
    
    def run(self):
        while threading.main_thread().is_alive():
            for key in self.tag_handlers:
                self.process_handler(key)
            time.sleep(0.5)
 
    def get_result(self, tag):
        result = None
        self.locker.acquire()
        if tag in self.tag_results:
            result = self.tag_results[tag]
        self.locker.release()
        return result

def now_us():
    now = datetime.now()
    timestamp = int(time.mktime(now.timetuple()))
    return timestamp * 1000000 + now.microsecond

def now_ms():
	return now_us() // 1000

def is_number(data):
    return re.match(r'^[-+]?[0-9]*\.?[0-9]+([eE][-+]?[0-9]+)?$', data) is not None

def is_integer(data):
    return re.match(r'^-?\d+$', data) is not None

def is_keys_in_dict(data, keys):
	for key in keys:	
		if key not in data:
			return False
	return True

def get_type_value_from_string(data):
    ret_value = data
    if is_number(data) == True:
        if is_integer(data) == True: 
            ret_value = int(data)
        else:
            ret_value = float(data)
    return ret_value

def run_cmd_with_result(cmd):
	result = ''
	try:
		result = subprocess.check_output(cmd, shell=True).decode()
	except Exception as e:
		pass
	return result

def split_field_by_space(data):
	fields = []
	arr = data.split(' ')
	for item in arr:
		if len(item) > 0:
			fields.append(item)
	return fields

def split_field_by_tab(data):
	fields = []
	arr = data.split('\t')
	for item in arr:
		if len(item) > 0:
			fields.append(item)
	return fields

def format_multi_line_text_to_map(data):
    results = {}
    arr = data.split('\n')
    for item in arr:
        if len(item) == 0:
            continue
        index = item.find(':')
        key   = item[0:index].strip()
        value = item[index+1:].strip()
        results[key] = get_type_value_from_string(value)
    return results

def format_kv_result(data, opts = {}):
    check_field_ok = None
    check_value_ok = None
    skip_line = 0
    skip_field_column = 0
    skip_value_column = 0

    if 'check_field_ok' in opts:
        check_field_ok = opts['check_field_ok']
    if 'check_value_ok' in opts:
        check_value_ok = opts['check_value_ok']
    if 'skip_line' in opts:
        skip_line = opts['skip_line']
    if 'skip_field_column' in opts:
        skip_field_column = opts['skip_field_column']
    if 'skip_value_column' in opts:
        skip_value_column = opts['skip_value_column']

    lines = data.split('\n')
    if len(lines) < 2:
        return None
    if skip_line > 0:
        lines = lines[skip_line:]
    arr = lines[0].split(' ')
    if skip_field_column > 0:
        arr = arr[skip_field_column:]
    fields = []
    values = []
    field_name  = ''
    field_value = ''
    field_count = 0
    check_field_id = 0
    check_value_id = 0
    for i in range(len(arr)):
        item = arr[i]
        if len(item) == 0:
            continue
        last_token = False
        if i == len(arr) - 1:
            last_token = True
        if check_field_ok is None:
            fields.append(item)
        else:
            if len(field_name) == 0:
                field_name = item
            else:
                field_name = field_name + ' ' + item
            if check_field_ok(check_field_id, field_name, last_token) == True:
                fields.append(field_name)
                field_name = '' 
                field_count = field_count + 1
                check_field_id = check_field_id + 1
    for line in lines[1:]:
        if len(line) == 0:
            continue
        line_values = []
        check_value_id = 0
        arr = line.split(' ')
        if skip_value_column > 0:
            arr = arr[skip_value_column:]
        for i in range(len(arr)):
            item = arr[i]
            if len(item) == 0:
                continue
            last_token = False
            if i == len(arr) - 1:
                last_token = True
            if check_value_ok is None:
                line_values.append(item)
            else:
                if len(field_value) == 0:
                    field_value = item
                else:
                    field_value = field_value + ' ' + item
                if check_value_ok(check_value_id, item, last_token) == True:
                    line_values.append(field_value)
                    field_value = ''
                    check_value_id = check_value_id + 1
        if len(fields) != len(line_values):
            return None
        values.append(line_values)
    results = []
    for i in range(len(values)):
        dt_result = {}
        for j in range(len(fields)):
            dt_result[fields[j]] = get_type_value_from_string(values[i][j])
        results.append(dt_result)
    return results 

def get_function_name():
	return inspect.currentframe().f_code.co_name

def get_file_content(path):
	content = None
	try:
		with open(path, 'r') as f:
			content = f.read()
	except Exception as e:
		print('get_file_content exception, path:{}, except:{}'.format(path, e))
	return content

####################################################################
#                       hardware info
####################################################################
def get_mem_detail_info():
    content = get_file_content('/proc/meminfo')
    if content is not None:
        data = format_multi_line_text_to_map(content)
        buffers   = get_type_value_from_string(data['Buffers'].split(' ')[0])
        cached    = get_type_value_from_string(data['Cached'].split(' ')[0])
        mem_free  = get_type_value_from_string(data['MemFree'].split(' ')[0])
        mem_total = get_type_value_from_string(data['MemTotal'].split(' ')[0])
        mem_used  = mem_total - mem_free - buffers - cached
        swap_total  = get_type_value_from_string(data['SwapTotal'].split(' ')[0])
        swap_free   = get_type_value_from_string(data['SwapFree'].split(' ')[0])
        swap_cached = swap_total - swap_free
        dirty       = get_type_value_from_string(data['Dirty'].split(' ')[0])
        write_back  = get_type_value_from_string(data['Writeback'].split(' ')[0])

        content = {
            'mem_total':   '{} Kb'.format(float(mem_total)),
            'mem_used':    '{} Kb'.format(float(mem_used)),
            'mem_free':    '{} Kb'.format(float(mem_free)),
            'buffers':     '{} Kb'.format(float(buffers)), 
            'cached':      '{} Kb'.format(float(cached)), 
            'swap_total':  '{} Kb'.format(float(swap_total)),
            'swap_cached': '{} Kb'.format(float(swap_cached)),
            'swap_free':   '{} Kb'.format(float(swap_free)),
            'dirty':       '{} Kb'.format(float(dirty)),
            'write_back':  '{} Kb'.format(float(write_back))
        }
    return content

def get_cpu_info():
	cmd_result = run_cmd_with_result('LANG=en_US lscpu')
	format_result = format_multi_line_text_to_map(cmd_result)
	results = {
		'count':     format_result['CPU(s)'],
		'modal':     format_result['Model name'],
		'mhz':       format_result['CPU MHz'],
		'L1d cache': format_result['L1d cache'],
		'L1i cache': format_result['L1i cache'],
		'L2 cache':  format_result['L2 cache'],
		'L3 cache':  format_result['L3 cache']
	}
	return results

def get_mem_info():
	mem_size = '-'
	mem_info = get_mem_detail_info()
	if mem_info is not None: 
		mem_size = mem_info['mem_total'] 
	return {'size': mem_size}

def check_df_field_is_fine(check_field_id, field_name, last_token = False):
    field_name_set = set()
    field_name_set = {'Filesystem', 'Size', 'Used', 'Avail', 'Use%', 'Mounted on'}
    if field_name not in field_name_set:
        return False
    return True

def check_df_value_is_fine(check_value_id, field_value, last_token = False):
    ret = True
    if check_value_id == 5 and last_token == False:
        ret = False
    return ret

def get_disk_info():
	disk_infos = []
	cmd_result = run_cmd_with_result('LANG=en_US df -h')
	disk_results = format_kv_result(
		data = cmd_result, 
		opts = {
			'check_field_ok': check_df_field_is_fine,
			'check_value_ok': check_df_value_is_fine
		}
	)
	for disk in disk_results:
		disk_infos.append({
			'mount':   disk['Mounted on'], 
			'fs_type': disk['Filesystem'],
			'total':   disk['Size'],
			'used':    disk['Used'],
			'avail':   disk['Avail'],
			'use%':    disk['Use%']
		})
	return disk_infos

def match_ip(text):
    pattern = re.compile(r'\b(?:[0-9]{1,3}\.){3}[0-9]{1,3}\b')
    return pattern.findall(text)

def match_mac(text):
    pattern = re.compile(r'\b(?:[0-9a-fA-F]{1,2}\:){5}[0-9a-fA-F]{1,2}\b')
    return pattern.findall(text)

def get_net_interface_names():
	names = []
	with os.scandir('/sys/class/net') as entries:
		for entry in entries:
			names.append(entry.name)
	return names

def get_net_detail_info(name):
    result = {
        'name':  name,
        'ip':    '-',
        'ipv6':  '-',
        'mac':   '-',
        'mtu':   '-',
        'speed': '-'
    }
    cmd_result = run_cmd_with_result('LANG=en_US ifconfig {}'.format(name))
    arr = cmd_result.split('\n')
    for item in arr:
        index = item.find('inet')
        if index != -1:
            key = 'ip'
            if item.find('inet6') != -1:
                key = 'ipv6'  
            ips = match_ip(item)
            if len(ips) > 0:
                result[key] = ips[0]
            
        index = item.find('ether')
        if index != -1:
            macs = match_mac(item)
            if len(macs) > 0:
                result['mac'] = macs[0]

        index = item.find('mtu ')
        if index != -1:
            result['mtu'] = item[index + 4:]

    speed = get_file_content('/sys/class/net/{}/speed'.format(name))
    if speed is not None:
        result['speed'] = '{}Mb'.format(speed.replace('\n', ''))
    return result 

def get_net_info():
	net_infos = []
	net_names = get_net_interface_names()
	for item in net_names:
		net_infos.append(get_net_detail_info(item))
	return net_infos

####################################################################
#                       system status
####################################################################

g_last_net_statistics = {}

def get_tcp_netstat_statistics():
    regex = re.compile(r'(\d+)')
    result1 = run_cmd_with_result('netstat -s')
    result2 = run_cmd_with_result('netstat -anp')
    arr = result1.split('\n')
    for item in arr:
        key = ''
        is_set = False
        if item.find('active connections') != -1:
            key = 'actives_per_sec'
        if item.find('passive connection') != -1:
            key = 'passives_per_sec'
        if item.find('retransmited') != -1:
            key = 'retrans_per_sec'
        if item.find('connections established') != -1:
            key = 'established'
            is_set = True
        if len(key) == 0:
            continue
        matches = regex.search(item)
        if matches:
            if key not in g_last_net_statistics:
                g_last_net_statistics[key] = int(matches.group())
            if is_set == True:
                g_last_net_statistics[key] = int(matches.group())
            else:
                g_last_net_statistics[key] = int(matches.group()) - g_last_net_statistics[key]
    time_wait = 0
    arr = result2.split('\n')
    for item in arr:
        if item.find('TIME_WAIT') != -1:
            time_wait = time_wait + 1	
    g_last_net_statistics['time_wait'] = time_wait
    return g_last_net_statistics

def get_softirq_status():
	global g_softirq_counter
	data = get_file_content('/proc/softirqs')
	if data is None:
		return None
	cpu_count = multiprocessing.cpu_count()
	softirq_counter = {}
	lines = data.split('\n')
	for item in lines[1:]:
		fields = split_field_by_space(item)
		if len(fields) == 0 or len(fields) <= cpu_count:
			continue
		irq_name  = ''
		cpu_usage = 0
		for i in range(cpu_count+1):
			if i == 0:
				irq_name = fields[0]
			else:
				cpu_usage = cpu_usage + int(fields[i])
		softirq_counter[irq_name.lower()] = cpu_usage
	if len(g_softirq_counter) == 0:
		g_softirq_counter = softirq_counter
	delta_softirq_counter = {}
	for k, v in softirq_counter.items():
		delta_softirq_counter[k] = v - g_softirq_counter[k]
	g_softirq_counter = softirq_counter
	return delta_softirq_counter

def is_kernel_thread(name):
	for item in g_kernel_thread_names:
		if name[0:1] == '[' and name.find(item) != -1:
			return True
	return False

def check_cpu_status_value_is_fine(check_value_id, field_value, last_token = False):
    ret = True
    if check_value_id == 10 and last_token == False:
        ret = False
    return ret

def get_cpu_status():
    global g_back_collector
    cmd_result = run_cmd_with_result('LANG=en_US ps aux')
    kv_result1 = g_back_collector.get_result('cpu_status.param1')
    if kv_result1 is None:
        return None
    kv_result2 = g_back_collector.get_result('cpu_status.param2')
    if kv_result2 is None:
        return None
    kv_result3 = format_kv_result(
        data = cmd_result, 
        opts = {'check_value_ok': check_cpu_status_value_is_fine}
    )
    all_cpu_status = {}
    arr_per_cpu_status = []
    if len(kv_result1) > 0 and len(kv_result2) > 0:
        all_cpu_status = {
            'usr%':     float(kv_result2[0]['%usr']),
            'sys%':     float(kv_result2[0]['%sys']),
            'iowait%':  float(kv_result2[0]['%iowait']),
            'hi%':      float(kv_result2[0]['%irq']),
            'si%':      float(kv_result2[0]['%soft']),
            'cs':       int(kv_result1[0]['cs']),
            'rqueues':  int(kv_result1[0]['r']),
            'bqueues':  int(kv_result1[0]['b'])
        }
        kv_result2 = kv_result2[1:]
        for i in range(len(kv_result2)):
            arr_per_cpu_status.append({
                'id':     i,
                'usr%':    float(kv_result2[i]['%usr']),
                'sys%':    float(kv_result2[i]['%sys']),
                'iowait%': float(kv_result2[i]['%iowait']),
                'hi%':     float(kv_result2[i]['%irq']),
                'si%':     float(kv_result2[i]['%soft'])
            })
    map_kernel_thread_status = {}
    for item in kv_result3:
        if is_kernel_thread(item['COMMAND']) == False:
            continue
        thread_name = item['COMMAND'][1:len(item['COMMAND'])-1]
        thread_type = thread_name.split('/')[0]
        if thread_type not in map_kernel_thread_status: 
            map_kernel_thread_status[thread_type] = {
                'cpu': 0.0,
                'threads': []
            }
        map_kernel_thread_status[thread_type]['cpu'] += float(item['%CPU'])
        map_kernel_thread_status[thread_type]['threads'].append({
            'pid': int(item['PID']),
            'cpu': float(item['%CPU'])
        })
    result = {
        'all_cpu': all_cpu_status, 
        'cpu_list': arr_per_cpu_status, 
        'kernel_thread': map_kernel_thread_status
    }
    return result

def get_mem_status():
	global g_back_collector
	kv_result1 = g_back_collector.get_result('mem_status.param1')
	if kv_result1 is None:
		return None
	kv_result2 = g_back_collector.get_result('mem_status.param2')
	if kv_result2 is None:
		return None
	kv_result3 = g_back_collector.get_result('mem_status.param3')
	if kv_result3 is None:
		return None
	kv_result4 = g_back_collector.get_result('mem_status.param4')
	if kv_result4 is None:
		return None
	mem_result  = {}	
	page_result = {}
	swap_result = {}
	for key in kv_result1[0]:
		mem_result[key] = kv_result1[0][key]
	for key in kv_result2[0]:
		page_result[key] = kv_result2[0][key]
	swap_result['pswpin/s']  = kv_result4[0]['pswpin/s']
	swap_result['pswpout/s'] = kv_result4[0]['pswpout/s']
	swap_result['kbswpfree'] = kv_result3[0]['kbswpfree']
	swap_result['kbswpused'] = kv_result3[0]['kbswpused']
	swap_result['%swpused']  = kv_result3[0]['%swpused']
	return {
		'mem':  mem_result,
		'page': page_result,
		'swap': swap_result
	}

def get_sar_result(flag, param = None):
	cmd = 'LANG=en_US sar -{}'.format(flag)
	if param is not None:
		cmd = '{} {}'.format(cmd, param)
	result = run_cmd_with_result(cmd)
	lines  = result.split('\n')
	if len(lines) < 5:
		return None
	lines = lines[2:]	
	result_lines = []
	result_data = ''
	for i in range(len(lines)):
		if i == 0 or lines[i].find('Average') != -1:
			result_data = result_data + lines[i]
			if i != len(lines):
				result_data = result_data + '\n'
	skip_field_column = 1
	skip_value_column = 1
	# centos sar time field has AM/PM part
	if lines[0].find(' AM ') != -1 or lines[0].find(' PM ') != -1:
		skip_field_column = skip_field_column + 1
	return format_kv_result(
		data = result_data, 
		skip_field_column = skip_field_column, 
		skip_value_column = skip_value_column
	)

def get_disk_status():
    global g_back_collector
    kv_results = g_back_collector.get_result('disk_status.param1')
    disk_results = []
    for item in kv_results:
        if item['Device:'].find('loop') != -1:
            continue
        item['device'] = item['Device:']
        del item['Device:']
        disk_results.append(item)
    return disk_results

def make_sar_net_obj(name):
	return {
		'iface':    name,
		'rxpck/s':  '-',
		'txpck/s':  '-',
		'rxkB/s':   '-',
		'txkB/s':   '-',
		'util%':    '-',
		'rxerr/s':  '-',
		'txerr/s':  '-',
		'rxdrop/s': '-',
		'txdrop/s': '-'
	}

def get_net_status():
	global g_back_collector
	sar_dev_results_map  = {}
	sar_edev_results_map = {}
	param1 = g_back_collector.get_result('net_status.param1')
	param2 = g_back_collector.get_result('net_status.param2')
	for item in param1:
		sar_dev_results_map[item['IFACE']] = item
	for item in param2:
		sar_edev_results_map[item['IFACE']] = item

	netstat_statistics = []
	for name in get_net_interface_names():
		sar_obj = make_sar_net_obj(name)
		if name in sar_dev_results_map:
			for k, v in sar_dev_results_map[name].items():
				if k == '%ifutil':
					sar_obj['util%'] = int(item['%ifutil'] * 100)
				elif k != 'IFACE':
					sar_obj[k] = v
		if name in sar_edev_results_map:
			for k, v in sar_edev_results_map[name].items():
				if k != 'IFACE':
					sar_obj[k] = v
		netstat_statistics.append(sar_obj)
	return netstat_statistics

def get_oom_status():
	oom_list = []
	result = run_cmd_with_result('LANG=en_US dmesg -T')
	for item in result.split('\n'):
		if len(item) == 0:
			continue
		if item.find('oom') == -1 or item.find('out of memory') == -1:
			continue
		beg_pos = item.find('[')
		end_pos = item.find(']')
		if beg_pos == -1 or end_pos == -1:
			continue
		time_str = item[beg_pos+1:end_pos-beg_pos]
		text_str = item[end_pos+1:]
		format_time_str = datetime.strptime(time_str, '%a %b %d %H:%M:%S %Y').strftime('%Y-%m-%d %H:%M:%S')
		oom_list.append({'time': time_str, 'data': text_str})
	cur_date = datetime.now().strftime('%Y-%m-%d')
	cur_date_oom_list = []
	result = {
		'last_oom':  {},
		'today_oom': [],
		'total_oom': []
	}
	if len(oom_list) > 0:
		result['last_oom'] = oom_list[-1:]
		for item in oom_list:	
			if item['time'].find(cur_date) != -1:
				result['today_oom'].append(item)
			result['total_oom'].append(item)
	return result

def get_system_status():
	result = run_cmd_with_result('uptime').replace('\n', '')
	index  = result.find('load average:') + 13
	arr = result[index:].split(',')
	cpu_load_results = {
		'last_1min':  get_type_value_from_string(arr[0].strip()),
		'last_5min':  get_type_value_from_string(arr[1].strip()),
		'last_15min': get_type_value_from_string(arr[2].strip())
	}
	results = {
		'cpu_load': cpu_load_results,
		'irq_soft': get_softirq_status(),
		'oom':      get_oom_status()
	}
	return CmdTagResult('system_status', results)

def get_name_info(name):
	ret_name = name
	ret_run_type = 'process'
	if name.find('|__') == 0:
		ret_name = name[3:]
		ret_run_type = 'thread'
	return ret_name, ret_run_type

def process_pidstat_result(result, keys):
	status = {}
	for item in result:
		if is_keys_in_dict(item, keys) == False:
			return None
		name, run_type = get_name_info(item['Command'])
		name_key = name + '-' + run_type
		obj = {
			'name': name,
			'type': run_type,
		}
		for key in keys:
			if key == 'Command':
				continue
			obj[key] = item[key]
		status[name_key] = obj

	process_status = {}
	threads_status = []
	for key in status:
		if status[key]['type'] == 'process':
			process_status = status[key]
		else:
			threads_status.append(status[key])
	result = {
		'process': process_status,
		'threads': threads_status
	}
	return result


def get_pid_cpu_status():
	global g_back_collector
	param1 = g_back_collector.get_result('cpu_status_pid.param1')
	param2 = g_back_collector.get_result('cpu_status_pid.param2')

	if param1 is None or param2 is None:
		return None

	status = {}
	for item in param1:
		if is_keys_in_dict(item, ['%usr', '%system', '%CPU', 'Command']) == False:
			return None
		name, run_type = get_name_info(item['Command'])
		name_key = name + '-' + run_type
		status[name_key] = {
			'name':    name,
			'type':    run_type,
			'%usr':    item['%usr'],
			'%system': item['%system'],
			'%total':  float(item['%usr']) + float(item['%system']),
			'%CPU':    item['%CPU']
		}

	for item in param2:
		if is_keys_in_dict(item, ['cswch/s', 'nvcswch/s', 'Command']) == False:
			return None
		name, run_type = get_name_info(item['Command'])
		name_key = name + '-' + run_type
		status[name_key].update({
			'cswch/s':   item['cswch/s'],
			'nvcswch/s': item['nvcswch/s']
		})

	process_status = {}
	threads_status = []
	for key in status:
		name = key.split('-')[0]
		run_type = key.split('-')[1]
		if run_type == 'process':
			process_status = status[key]
		else:
			threads_status.append(status[key])
	result = {
		'process': process_status,
		'threads': threads_status
	}
	return result

#def get_pid_mem_status():
#	global g_back_collector
#	param = g_back_collector.get_result('mem_status_pid.param1')
#
#	status = []
#	for item in param:
#		if is_keys_in_dict(item, ['minflt/s','majflt/s','VSZ','RSS','%MEM','Command']):
#			return None
#		name, run_type = get_name_info(item['Command'])
#		status.append({
#			'name':     name,	
#			'type':     run_type,
#			'minflt/s': item['minflt/s'],
#			'majflt/s': item['majflt/s'],
#			'VSZ':		item['VSZ'],
#			'RSS':		item['RSS'],
#			'%MEM':		item['%MEM']
#		})
#
#	process_status = {}
#	threads_status = []
#	for key in status:
#		if status[key]['type'] == 'process':
#			process_status = status[key]
#		else:
#			threads_status.append(status[key])
#	result = {
#		'process': process_status,
#		'threads': threads_status
#	}
#	print('++++++++++ pid_mem_status:{}'.format(result))
#	return result

def get_pid_mem_status():
	global g_back_collector
	result = g_back_collector.get_result('mem_status_pid.param1')
	return process_pidstat_result(result, ['minflt/s','majflt/s','VSZ','RSS','%MEM','Command'])

#def get_pid_disk_status():
#	global g_back_collector
#	param = g_back_collector.get_result('disk_status_pid.param1')
#
#	status = []	
#	for item in param:
#		if is_keys_in_dict(item, ['kB_rd/s', 'kB_wr/s', 'kB_ccwr/s', 'Command']):
#			return None
#		name, run_type = get_name_info(item['Command'])
#		status.append({
#			'name':      name,
#			'type':      run_type,
#			'kB_rd/s':   item['kB_rd/s'],
#			'kB_wr/s':	 item['kB_wr/s'],
#			'kB_ccwr/s': item['kB_ccwr/s']
#		})
#
#	process_status = {}
#	threads_status = []
#	for key in status:
#		if status[key]['type'] == 'process':
#			process_status = status[key]
#		else:
#			threads_status.append(status[key]);
#	result = {
#		'process': process_status,
#		'threads': threads_status
#	}
#	return result

def get_pid_disk_status():
	global g_back_collector
	result = g_back_collector.get_result('disk_status_pid.param1')
	return process_pidstat_result(result, ['kB_rd/s', 'kB_wr/s', 'kB_ccwr/s', 'Command'])

def check_pn_field_is_fine(check_field_id, field_name, last_token = False):
	field_name_set = set()
	field_name_set = {'Proto', 'Recv-Q', 'Send-Q', 'Local Address', 'Foreign Address', 'State', 'PID/Program name'}
	if field_name not in field_name_set:
		return False
	return True	

def server_status_key_value_set(server_status, port, key, value):
	if port not in server_status:
		server_status[port] = {}
	server_status[port][key] = value

def server_status_key_value_incr(server_status, port, key, value):
	if port not in server_status:
		server_status[port] = {}
	if key not in server_status[port]:
		server_status[port][key] = value;
	else:
		server_status[port][key] += value;

def dict_multi_key_value_set(dt, keys, value):
	last_key = keys[-1]
	dt_ptr = dt;
	for key in keys:
		if key not in dt_ptr and key != last_key:
			dt_ptr[key] = {}
		if key == last_key:
			dt_ptr[key] = value
		dt_ptr = dt_ptr[key]
	return dt

def dict_multi_key_value_incr(dt, keys, value):
	last_key = keys[-1]
	dt_ptr = dt
	for key in keys:
		if key not in dt_ptr and key != last_key:
			dt_ptr[key] = {}
		if key == last_key:
			if key in dt_ptr:
				dt_ptr[key] = dt_ptr[key] + value
			else:
				dt_ptr[key] = value
		dt_ptr = dt_ptr[key]
	return dt

def get_ss_format_kv_result(result):
	new_lines = []
	lines = result.split('\n')
	for line in lines:
		if len(line) == 0:
			continue
		items = split_field_by_space(line)	
		if len(items) != 6:
			continue
		new_lines.append({
			'state':  items[0],
			'pid':    items[5].split(',')[1].split('=')[1],       
			'recv-q': items[1],
			'send-q': items[2],
			'local_addr': items[3],
			'peer_addr':  items[4]
		})
	return new_lines	

def get_pid_net_flow_rate(pid):
	global g_back_collector
	net_flow_rate = {}
	result = g_back_collector.get_result('net_status_pid.param1')
	if result is not None:
		for line in result.split('\n'):
			if len(line) == 0:
				continue
			index = line.find('/{}/'.format(pid))
			if index == -1:
				continue
			items = split_field_by_tab(line)
			if len(items) != 3:
				continue
			net_flow_rate = {
				'send': items[1],
				'recv': items[2]
			}
			break
	return net_flow_rate	

def get_pid_net_status(pid):
	cmd_result1 = run_cmd_with_result('ss -ltnp | grep "pid={}"'.format(pid))
	cmd_result2 = run_cmd_with_result('ss -tnp | grep "pid={}"'.format(pid))

	if cmd_result1 == '' and cmd_result2 == '':
		return None

	result1 = get_ss_format_kv_result(cmd_result1)
	result2 = get_ss_format_kv_result(cmd_result2)

	listen_addrs  = set()
	client_status = {}
	server_status = {}

	for item in result1:
		listen_addrs.add(item['local_addr'])
		server_status = dict_multi_key_value_set(server_status, [item['local_addr'], 'accept_list'], int(item['recv-q']))
		server_status = dict_multi_key_value_set(server_status, [item['local_addr'], 'max_accept_list'], int(item['send-q']))

	for item in result2:
		addr  = item['local_addr']
		state = item['state']
		any_addr = '*:{}'.format(addr.split(':')[1])
		if addr in listen_addrs or any_addr in listen_addrs:
			refer_addr = ''
			if addr in listen_addrs:
				refer_addr = addr
			else:
				refer_addr = any_addr
			# 服务端统计
			server_status = dict_multi_key_value_incr(server_status, [refer_addr, 'listen_list'], 0)
			if state == 'SYN_RECV':
				server_status = dict_multi_key_value_incr(server_status, [refer_addr, 'listen_list'], 1)
			elif state == 'ESTAB':
				server_status = dict_multi_key_value_incr(server_status, [refer_addr, 'connects'], 1)
			server_status = dict_multi_key_value_incr(server_status, [refer_addr, 'socket_state', state], 1)
		else:	
			# 客户端统计
			if state == 'ESTAB':
				client_status = dict_multi_key_value_incr(client_status, [item['peer_addr'], 'connects'], 1)
			client_status = dict_multi_key_value_incr(client_status, [item['peer_addr'], 'socket_state', state], 1)

	new_client_status = []
	new_server_status = []

	for key in client_status:
		obj = client_status[key]
		obj['addr'] = key
		new_client_status.append(obj)
	for key in server_status:
		obj = server_status[key]
		obj['addr'] = key
		new_server_status.append(obj)
	result =  {
		'client_status': new_client_status, 
		'server_status': new_server_status,
		'flow_rate':     get_pid_net_flow_rate(pid)
	}
	return result


def print_info():
	cpu_result = get_cpu_info()
	mem_result = get_mem_info()
	net_result  = get_net_info()
	disk_result = get_disk_info()

	print('\033[32mCPU:\033[0m')
	print('modal:{}'.format(cpu_result['modal']))
	print('count:{}'.format(cpu_result['count']))
	print('L1d cache:{}'.format(cpu_result['L1d cache']))
	print('L1i cache:{}'.format(cpu_result['L1i cache']))
	print('L2 cache:{}'.format(cpu_result['L2 cache']))
	print('L3 cache:{}'.format(cpu_result['L3 cache']))
	print('')
	print('\033[32mMEM:\033[0m')
	print('size:{}'.format(mem_result['size']))
	print('')
	print('\033[32mNET:\033[0m')
	print('{} {} {} {} {}'.format(
		format('name', '<15'),
		format('ip', '<15'),
		format('mac', '<20'),
		format('mtu', '<10'),
		format('speed', '<10')
	))
	for item in net_result:
		print('{} {} {} {} {}'.format(
			format(item['name'], '<15'),
			format(item['ip'], '<15'),
			format(item['mac'], '<20'),
			format(item['mtu'], '<10'),
			format(item['speed'], '<10')
		))
	print('')
	print('\033[32mDISK:\033[0m')
	max_fs_type_size = 0
	for item in disk_result:
		size = len(item['fs_type'])	
		if size > max_fs_type_size:
			max_fs_type_size = size
	print('{} {} {} {} {}'.format(
		format('fs_type', '<{}'.format(max_fs_type_size)),
		format('total', '<{}'.format(10)),
		format('used', '<{}'.format(10)),
		format('avail', '<{}'.format(10)),
		format('use%', '<{}'.format(10))
	))
	for item in disk_result:
		print('{} {} {} {} {}'.format(
			format(item['fs_type'], '<{}'.format(max_fs_type_size)),
			format(item['total'], '<{}'.format(10)),
			format(item['used'], '<{}'.format(10)),
			format(item['avail'], '<{}'.format(10)),
			format(item['use%'], '<{}'.format(10))
		))
	print('')


class FormatResultBlank:
	def process(self, data, opt):
		return data


class FormatResultCommon:
	def __init__(self, config):
		self.config = config
		self.set_title = ''

	def process(self, data, opt):
		lines = data.split('\n')
		if 'first_result' in opt and opt['first_result'] == True:
			if 'first_result_ignore' in self.config and self.config['first_result_ignore'] == True:
				return None
			if 'first_result_skip_line' in self.config:
				lines = lines[self.config['first_result_skip_line']:]
		opts = {}
		if 'skip_column' in self.config:
			opts = {
				'skip_field_column': self.config['skip_column'],
				'skip_value_column': self.config['skip_column']
			}
		lines = self.process_title(lines)
		new_data = ''
		for item in lines:
			if len(item) > 0:
				new_data = new_data + item + '\n'
		return format_kv_result(new_data, opts)

	def process_title(self, lines):
		if 'set_title' not in self.config:
			return lines
		with_title = False
		new_lines = []
		for line in lines:
			is_title = True
			for item in self.config['set_title']:			
				if line.find(item) == -1:
					is_title = False	
					break
			if is_title == True:
				with_title = True
				if len(self.set_title) == 0:
					self.set_title = line
			new_lines.append(line)
		if with_title == False:
			new_lines = [self.set_title] + new_lines
		return new_lines


class FormatResultCpuStatusParam1:
	def __init__(self):
		self.title = None

	def process(self, data, opt):
		new_data = ''
		lines = data.split('\n')
		for item in lines:
			if len(item) == 0 or item.find('memory') != -1:
				continue
			if item.find('swpd') != -1:
				if self.title is None:
					self.title = item
				continue
			new_data = self.title + '\n' + item
			break
		if len(new_data) == 0:
			return None
		return format_kv_result(new_data)


def print_status():
	global g_back_collector
	g_back_collector = CmdOutputCollector()
	g_back_collector.start()

	ret, msg = g_back_collector.add_handle(
		tag='cpu_status.param1', 
		cmd='LANG=en_US vmstat 1', 
		handler=FormatResultCpuStatusParam1()
	)
	if ret == False:
		print('add_handle:cpu_status.param1 fail, error:{}'.format(msg))
		return
	ret, msg = g_back_collector.add_handle(
		tag='cpu_status.param2', 
		cmd='LANG=en_US mpstat -P ALL 1',
		handler=FormatResultCommon({
			'first_result_ignore': True, 
			'skip_column': 2
		})
	)
	if ret == False:
		print('add_handle:cpu_status.param2 fail, error:{}'.format(msg))
		return
	ret, msg = g_back_collector.add_handle(
		tag='cpu_status.param3',
		cmd='LANG=en_US pidstat -w 1',
		handler=FormatResultCommon({
			'first_result_ignore': True, 
			'skip_column': 2
		})
	)
	if ret == False:
		print('add_handle:cpu_status.param3 fail, error:{}'.format(msg))
		return

	ret, msg = g_back_collector.add_handle(
		tag='mem_status.param1',
		cmd='LANG=en_US sar -r 1',
		handler=FormatResultCommon({
			'first_result_ignore': True, 
			'skip_column': 2, 
			'set_title': ['kbmemfree', 'kbmemused']
		})
	)
	if ret == False:
		print('add_handle:mem_status.param1 fail, error:{}'.format(msg))
		return

	ret, msg = g_back_collector.add_handle(
		tag='mem_status.param2',
		cmd='LANG=en_US sar -B 1',
		handler=FormatResultCommon({
			'first_result_ignore': True,
			'skip_column': 2,
			'set_title': ['pgpgin/s', 'pgpgout/s']
		})
	)
	if ret == False:
		print('add_handle:mem_status.param2 fail, error:{}'.format(msg))
		return

	ret, msg = g_back_collector.add_handle(
		tag='mem_status.param3',
		cmd='LANG=en_US sar -S 1',
		handler=FormatResultCommon({
			'first_result_ignore': True,
			'skip_column': 2,
			'set_title': ['kbswpfree', 'kbswpused']
		})
	)
	if ret == False:
		print('add_handle:mem_status.param3 fail, error:{}'.format(msg))
		return

	ret, msg = g_back_collector.add_handle(
		tag='mem_status.param4',
		cmd='LANG=en_US sar -W 1',
		handler=FormatResultCommon({
			'first_result_ignore': True,
			'skip_column': 2,
			'set_title': ['pswpin/s', 'pswpout/s']
		})
	)
	if ret == False:
		print('add_handle:mem_status.param4 fail, error:{}'.format(msg))
		return

	ret, msg = g_back_collector.add_handle(
		tag='net_status.param1',
		cmd='LANG=en_US sar -n DEV 1',
		handler=FormatResultCommon({'first_result_ignore': True, 'skip_column': 2})
	)
	if ret == False:
		print('add_handle:net_status.param1 fail, error:{}'.format(msg))
		return
	ret, msg = g_back_collector.add_handle(
		tag='net_status.param2', 
		cmd='LANG=en_US sar -n EDEV 1',
		handler=FormatResultCommon({'first_result_ignore': True, 'skip_column': 2})
	)
	if ret == False:
		print('add_handle:net_status.param2 fail, error:{}'.format(msg))
		return

	ret, msg = g_back_collector.add_handle(
		tag='disk_status.param1', 
		cmd='LANG=en_US iostat -dx 1',
		handler=FormatResultCommon({'first_result_ignore': True})
	)
	if ret == False:
		print('add_handle:disk_status.param1 fail, error:{}'.format(msg))
		return

	while True:
		time.sleep(1)

		cpu_status = get_cpu_status()
		if cpu_status is None:
			continue

		mem_status = get_mem_status()
		if mem_status is None:
			continue

		net_status = get_net_status()
		if net_status is None:
			continue

		disk_status = get_disk_status()
		if disk_status is None:
			continue

		os.system('clear')

		print('\033[32mCPU:\033[0m')
		print('{} {} {} {} {} {} {} {} {}'.format(
			format('id', '<10'),
			format('usr%', '<10'),
			format('sys%', '<10'),
			format('iowait%', '<10'),
			format('hi%', '<10'),
			format('si%', '<10'),
			format('cs', '<10'),
			format('rqueues', '<10'),
			format('bqueues', '<10')
		))
		all_cpu = cpu_status['all_cpu']
		print('{} {} {} {} {} {} {} {} {}'.format(
			format('all', '<10'),
			format(all_cpu['usr%'],    '<10'),
			format(all_cpu['sys%'],    '<10'),
			format(all_cpu['iowait%'], '<10'),
			format(all_cpu['hi%'],     '<10'),
			format(all_cpu['si%'],     '<10'),
			format(all_cpu['cs'],      '<10'),
			format(all_cpu['rqueues'], '<10'),
			format(all_cpu['bqueues'], '<10'),
			format('-', '<10'),
			format('-', '<10'),
			format('-', '<10')
		))
		for item in cpu_status['cpu_list']:
			print('{} {} {} {} {} {} {} {} {}'.format(
				format(item['id'], '<10'),
				format(item['usr%'], '<10'),
				format(item['sys%'], '<10'),
				format(item['iowait%'], '<10'),
				format(item['hi%'], '<10'),
				format(item['si%'], '<10'),
				format('-', '<10'),
				format('-', '<10'),
				format('-', '<10')
			))
		print('')

		print('\033[32mMEM:\033[0m')
		print('MEM:  {} {} {} {} {} {} {} {}'.format(
			format('mem_free',  '<10'),
			format('mem_used',  '<10'),
			format('%mem_used', '<10'),
			format('buffers',   '<10'),
			format('cached',    '<10'),
			format('active',    '<10'),
			format('inactive',  '<10'),
			format('dirty',     '<10')
		))
		print('      {} {} {} {} {} {} {} {}'.format(
			format(mem_status['mem']['kbmemfree'], '<10'),
			format(mem_status['mem']['kbmemused'], '<10'),
			format(mem_status['mem']['%memused'],  '<10'),
			format(mem_status['mem']['kbbuffers'], '<10'),
			format(mem_status['mem']['kbcached'],  '<10'),
			format(mem_status['mem']['kbactive'],  '<10'),
			format(mem_status['mem']['kbinact'],   '<10'),
			format(mem_status['mem']['kbdirty'],   '<10')
		))
		print('')
		print('PAGE: {} {} {} {} {} {} {} {}'.format(
			format('pgpgin/s',  '<10'),
			format('pgpgout/s', '<10'),
			format('fault/s',   '<10'),
			format('majflt/s',  '<10'),
			format('pgfree/s',  '<10'),
			format('pgscank/s', '<10'),
			format('pgscand/s', '<10'),
			format('pgsteal/s', '<10'),
			format('%vmeff',    '<10')
		))
		print('      {} {} {} {} {} {} {} {}'.format(
			format(mem_status['page']['pgpgin/s'],  '<10'),
			format(mem_status['page']['pgpgout/s'], '<10'),
			format(mem_status['page']['fault/s'],   '<10'),
			format(mem_status['page']['majflt/s'],  '<10'),
			format(mem_status['page']['pgfree/s'],  '<10'),
			format(mem_status['page']['pgscank/s'], '<10'),
			format(mem_status['page']['pgscand/s'], '<10'),
			format(mem_status['page']['pgsteal/s'], '<10'),
			format(mem_status['page']['%vmeff'],    '<10')
		))
		print('')
		print('SWAP: {} {} {} {} {}'.format(
			format('pswpin/s',  '<10'),
			format('pswpout/s', '<10'),
			format('kbswpfree', '<10'),
			format('kbswpused', '<10'),
			format('%swpused',  '<10')
		))
		print('      {} {} {} {} {}'.format(
			format(mem_status['swap']['pswpin/s'],  '<10'),
			format(mem_status['swap']['pswpout/s'], '<10'),
			format(mem_status['swap']['kbswpfree'], '<10'),
			format(mem_status['swap']['kbswpused'], '<10'),
			format(mem_status['swap']['%swpused'],  '<10')
		))
		print('')
		print('\033[32mNET:\033[0m')
		iface_size = 10
		for item in net_status:
			if len(item['iface']) > iface_size:
				iface_size = len(item['iface'])
		print('{} {} {} {} {} {} {} {} {} {}'.format(
			format('iface',    '<{}'.format(iface_size)),
			format('rxpck/s',  '<10'),
			format('txpck/s',  '<10'),
			format('rxkB/s',   '<10'),
			format('txkB/s',   '<10'),
			format('rxdrop/s', '<10'),
			format('txdrop/s', '<10'),
			format('rxfifo/s', '<10'),
			format('txfifo/s', '<10'),
			format('%util',    '<10')
		))
		for item in net_status:	
			print('{} {} {} {} {} {} {} {} {} {}'.format(
				format(item['iface'],    '<{}'.format(iface_size)),
				format(item['rxpck/s'],  '<10'),
				format(item['txpck/s'],  '<10'),
				format(item['rxkB/s'],   '<10'),
				format(item['txkB/s'],   '<10'),
				format(item['rxdrop/s'], '<10'),
				format(item['txdrop/s'], '<10'),
				format(item['rxfifo/s'], '<10'),
				format(item['txfifo/s'], '<10'),
				format(item['util%'],    '<10')
			))
		print('')
		print('\033[32mDISK:\033[0m')
		print('{} {} {} {} {} {} {} {} {} {} {} {} {} {}'.format(
			format('device',   '<10'),
			format('rrqm/s',   '<10'),
			format('wrqm/s',   '<10'),
			format('r/s',      '<10'),
			format('w/s',      '<10'),
			format('rkB/s',    '<10'),
			format('wkB/s',    '<10'),
			format('avgrq-sz', '<10'),
			format('avgqu-sz', '<10'),
			format('await',    '<10'),
			format('r_await',  '<10'),
			format('w_await',  '<10'),
			format('svctm',    '<10'),
			format('%util',    '<10')
		))
		for item in disk_status:
			print('{} {} {} {} {} {} {} {} {} {} {} {} {} {}'.format(
				format(item['device'],   '<10'),
				format(item['rrqm/s'],   '<10'),
				format(item['wrqm/s'],   '<10'),
				format(item['r/s'],      '<10'),
				format(item['w/s'],      '<10'),
				format(item['rkB/s'],    '<10'),
				format(item['wkB/s'],    '<10'),
				format(item['avgrq-sz'], '<10'),
				format(item['avgqu-sz'], '<10'),
				format(item['await'],    '<10'),
				format(item['r_await'],  '<10'),
				format(item['w_await'],  '<10'),
				format(item['svctm'],    '<10'),
				format(item['%util'],    '<10')
			))
		print('')	

def get_name_size(process, threads):
	size = len(process['name'])
	for item in threads:
		tsize = len(item['name'])
		if tsize > size:
			size = tsize
	return size + 1

def print_pid_status(pid):
	# cpu:
	# pidstat -t -p 77985 1

	# 上下文切换:
	# pidstat -w -t -p 78166 1

	# 内存占用:
	# pidstat -r -t -p 78166 1

	# 磁盘使用:
	# pidstat -d -t -p 79532 1

	# 网络状况(区分客户端、服务端):
	# (1) 客户端:
	#     1) socket连接数信息，包括：连接数、处理TIME_WAIT状态统计
	# (2) 服务端:
	#     1) socket连接数信息，包括：连接数、处理TIME_WAIT状态统计
	#     2) 半队列数，是否满
	#     3) 全队列数，是否满

	global g_back_collector
	g_back_collector = CmdOutputCollector()
	g_back_collector.start()

	# cpu状态信息
	ret, msg = g_back_collector.add_handle(
		tag='cpu_status_pid.param1',
		cmd='LANG=en_US pidstat -t -p {} 1'.format(pid),
		handler=FormatResultCommon({
			'first_result_ignore': True,
			'skip_column': 2,
			'set_title': ['%usr', '%system', 'Command']
		})
	)
	if ret == False:
		print('add_handle:cpu_status_pid.param1 fail, error:{}'.format(msg));
		return

	ret, msg = g_back_collector.add_handle(
		tag='cpu_status_pid.param2',
		cmd='LANG=en_US pidstat -w -t -p {} 1'.format(pid),
		handler=FormatResultCommon({
			'first_result_ignore': True,
			'skip_column': 2,
			'set_title': ['cswch/s', 'nvcswch/s', 'Command']
		})
	)
	if ret == False:
		print('add_handle:cpu_status_pid.param2 fail, error:{}'.format(msg))
		return

	# 内存占用
	ret, msg = g_back_collector.add_handle(
		tag='mem_status_pid.param1',
		cmd='pidstat -r -t -p {} 1'.format(pid),
		handler=FormatResultCommon({
			'first_result_ignore': True,
			'skip_column': 2,
			'set_title': ['minflt/s', 'majflt/s', 'Command']
		})
	)
	if ret == False:
		print('add_handle.mem_status_pid.param1 fail, error:{}'.format(msg))
		return

	# 磁盘使用
	ret, msg = g_back_collector.add_handle(
		tag='disk_status_pid.param1',
		cmd='pidstat -d -t -p {} 1'.format(pid),
		handler=FormatResultCommon({
			'first_result_ignore': True,
			'skip_column': 2,
			'set_title': ['kB_rd/s', 'kB_wr/s', 'kB_ccwr/s', 'Command']
		})
	)
	if ret == False:
		print('add_handle.disk_status_pid.param1 fail, error:{}'.format(msg))
		return

	# 网络使用
	ret, msg = g_back_collector.add_handle(
		tag='net_status_pid.param1',
		cmd='nethogs -d 1 -a -t'.format(pid),
		handler=FormatResultBlank()
	)
	if ret == False:
		print('add_handle.net_status_pid.param1 fail, error:{}'.format(msg))
		return


	while True:	
		time.sleep(1)

		cpu_status = get_pid_cpu_status()
		if cpu_status is None:
			continue

		mem_status = get_pid_mem_status()
		if mem_status is None:
			continue

		disk_status = get_pid_disk_status()
		if disk_status is None:
			continue

		net_status = get_pid_net_status(pid)
		if net_status is None:
			continue

		os.system('clear')

		print('\033[32mCPU:\033[0m')
		cpu_process = cpu_status['process']
		cpu_threads = cpu_status['threads']
		size = get_name_size(cpu_process, cpu_threads)
		print('\033[33m{} {} {} {} {} {} {}\033[0m'.format(
			format('name',      '<{}'.format(size)),
			format('type',      '<10'),
			format('%usr',      '<10'),
			format('%system',   '<10'),
			format('%total',    '<10'),
			format('cswch/s',   '<10'),
			format('nvcswch/s', '<10')
		))
		print('{} {} {} {} {} {} {}'.format(
			format(cpu_process['name'],      '<{}'.format(size)),
			format(cpu_process['type'],      '<10'),
			format(cpu_process['%usr'],      '<10'),
			format(cpu_process['%system'],   '<10'),
			format(cpu_process['%total'],    '<10'),
			format(cpu_process['cswch/s'],   '<10'),
			format(cpu_process['nvcswch/s'], '<10')
		))
		for item in cpu_threads:
			print('{} {} {} {} {} {} {}'.format(
				format(item['name'],      '<{}'.format(size)),
				format(item['type'],      '<10'),
				format(item['%usr'],      '<10'),
				format(item['%system'],   '<10'),
				format(item['%total'],    '<10'),
				format(item['cswch/s'],   '<10'),
				format(item['nvcswch/s'], '<10')
			))
		print('')

		print('\033[32mMEM:\033[0m')
		mem_process = mem_status['process']
		mem_threads = mem_status['threads']
		size = get_name_size(mem_process, mem_threads)
		print('\033[33m{} {} {} {} {}\033[0m'.format(
			format('name',     '<{}'.format(size)),
			format('type',     '<10'),
			format('minflt/s', '<10'),
			format('majflt/s', '<10'),
			format('VSZ',      '<10'),
			format('RSS',      '<10'),
			format('%MEM',     '<10')
		))
		print('{} {} {} {} {}'.format(
			format(mem_process['name'],     '<{}'.format(size)),
			format(mem_process['type'],     '<10'),
			format(mem_process['minflt/s'], '<10'),
			format(mem_process['majflt/s'], '<10'),
			format(mem_process['VSZ'],      '<10'),
			format(mem_process['RSS'],      '<10'),
			format(mem_process['%MEM'],     '<10')
		))
		for item in mem_threads:
			print('{} {} {} {} {}'.format(
				format(item['name'],     '<{}'.format(size)),
				format(item['type'],     '<10'),
				format(item['minflt/s'], '<10'),
				format(item['majflt/s'], '<10'),
				format(item['VSZ'],      '<10'),
				format(item['RSS'],      '<10'),
				format(item['%MEM'],     '<10')
			))
		print('')

		print('\033[32mDISK:\033[0m')
		disk_process = disk_status['process']
		disk_threads = disk_status['threads']
		size = get_name_size(disk_process, disk_threads)
		print('\033[33m{} {} {} {} {}\033[0m'.format(
			format('name',      '<{}'.format(size)),
			format('type',      '<10'),
			format('kB_rd/s',   '<10'),
			format('kB_wr/s',   '<10'),
			format('kB_ccwr/s', '<10')
		))
		print('{} {} {} {} {}'.format(
			format(disk_process['name'],      '<{}'.format(size)),
			format(disk_process['type'],      '<10'),
			format(disk_process['kB_rd/s'],   '<10'),
			format(disk_process['kB_wr/s'],   '<10'),
			format(disk_process['kB_ccwr/s'], '<10')
		))
		for item in disk_threads:
			print('{} {} {} {} {}'.format(
				format(item['name'],      '<{}'.format(size)),
				format(item['type'],      '<10'),
				format(item['kB_rd/s'],   '<10'),
				format(item['kB_wr/s'],   '<10'),
				format(item['kB_ccwr/s'], '<10')
			))
		print('')

		print('\033[32mNET:\033[0m')
		client_status = net_status['client_status']
		server_status = net_status['server_status']
		flow_rate     = net_status['flow_rate']
		print('\033[33mclient:\033[0m')
		print('[')
		for item in client_status:
			print('{')
			print('\taddr: {}'.format(item['addr']))
			print('\tstate:{')
			size = 0
			for key in item['socket_state']:
				if len(key) > size:
					size = len(key)
			for key in item['socket_state']:
				print('\t\t{}: {}'.format(
					format(key, '<{}'.format(size + 1)),
					item['socket_state'][key]
				))
			print('\t}')
			print('}')
		print(']')
		print('')
		print('\033[33mserver:\033[0m')
		print('[')
		for item in server_status:
			print('{')
			print('\taddr: {}'.format(item['addr']))
			print('\tlisten_queue: {{cur: {}}}'.format(item['listen_list']))
			print('\taccept_queue: {{cur: {}, max: {}}}'.format(
				item['accept_list'], 
				item['max_accept_list']
			))
			print('\tconnects: {}'.format(item['connects']))
			print('\tstate:{')
			size = 0
			for key in item['socket_state']:
				if len(key) > size:
					size = len(key)	
			for key in item['socket_state']:
				print('\t\t{}: {}'.format(
					format(key, '<{}'.format(size + 1)),
					item['socket_state'][key]
				))
			print('\t}')
			print('}')
		print(']')

		send_flow = '-'
		recv_flow = '-'
		if len(flow_rate) > 0:
			send_flow = str(flow_rate['send'])
			recv_flow = str(flow_rate['recv'])
		print('')
		print('\033[33mflow_rate:\033[0m')
		print('{')
		print('\tsend: {}'.format(send_flow))
		print('\trecv: {}'.format(recv_flow))
		print('}')


def print_field_instruct():
	print('cswch/s: 	每秒自愿上下文切换次数')
	print('nvcswch/s:	每秒非自愿上下文切换次数，引发原因有：时间片用完，进程争抢CPU，中断导致进程被切换')
	print('pgpgin/s:    每秒从磁盘或者SWAP置换到内存数')
	print('pgpgout/s:   每秒从内存置换到磁盘或者SWAP数，swappiness配置值表示当内存占用超过指定百分比后进行置换')
	print('pgscank/s:   每秒kswapd内核线程扫描页面数（必要时才会启动扫描获取空闲页面供使用）')
	print('pgscand/s:   每秒直接扫描页面数')
	print('pgsteal/s:   每秒从cache中被清除来满足内存需要的页个数')
	print('%vmeff:      pgsteal/(pgscank + pgscand)，衡量页面回收效率的指标。高数值意味着成功地从非活动列表中回收页面（健康），低数值表示系统在挣扎中，100%是高数值，小于30%是低数值')
	print('kbswpcad:	高速缓存交换空间：它同时保存在主存储器和交换设备中，因此不需要磁盘IO就能被换出页面')
	print('rxdrop/s:	当由于缓冲区满的时候，网卡设备接收端每秒钟丢掉的网络包的数目')
	print('txdrop/s:	当由于缓冲区满的时候，网卡设备发送端每秒钟丢掉的网络包的数目')
	print('rxfifo/s:	在接收数据包的时候，每秒缓冲区溢出的错误发生的次数')
	print('txfifo/s:	在发送数据包的时候，每秒缓冲区溢出的错误发生的次数')
	print('rrqm/s:		每秒合并放入驱动请求队列的读请求数')
	print('wrqm/s:		每秒合并放入驱动请求队列的写请求数')
	print('r/s:			每秒发给磁盘设备的读请求数')
	print('w/s:			每秒发给磁盘设备的写请求数')
	print('avgrq-sz:	平均请求大小，单位为扇区（512B）')
	print('avgqu-sz:	平均请求队列长度')
	print('await:		平均I/O响应时间，包括在驱动请求队列里等待何设备的I/O响应时间（ms）')
	print('svctm:		（推断）磁盘设备的I/O平均响应时间（ms）')
	print('%util:		设备忙处理I/O请求的百分比（使用率）')



if __name__ == '__main__':

	show_info = False
	show_status = False
	pid = None

	opts, args = getopt.getopt(sys.argv[1:], 'p:', ['info', 'status'])
	for key, value in opts:
		if key == '-p':
			pid = value
		elif key == '--info':
			show_info = True
		elif key == '--status':
			show_status = True

	if show_info == True:
		print_info()

	if show_status == True:
		if pid is None:
			print_status()
		else:
			print_pid_status(pid)


'''
kswapd内核线程: 
在内存不足的场景下，操作系统会唤醒kswapd内核线程回收一定的内存页（已映射但是暂时没有被用到）置换到swap中，并将回收的内存页防止空闲内存页链表供使用。kswapd有两个阈值pages_high、pages_low，当空闲内存页数量低于pages_low时就会扫描内存并释放页面，直到空闲页面数量达到pages_high

页面回收机制：
1、周期性回收内存机制：
这是kswapd内核线程的工作职责，当内核路径调用alloc_pages()分配物理页面时，由于系统内存短缺，没法在低水位情况下分配内存，因此会唤醒kswapd内核线程来异步回收内存

2、直接内存回收机制：
在内核态里调用页面分配接口函数alloc_pages()分配物理页面时，由于系统内存短缺，不能满足分配请求，内核会直接自陷到页面回收机制，尝试回收内存来解决当前的燃眉之急（即同步等待方式回收内存）

回收内存包括回收slab对象，slab分配的内存主要用于slab对象和kmalloc接口


'''

